package net.jxtz.jxtzos.spider.process;


import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import net.jxtz.jxtzos.entity.spider.SpiderHousingInformation;
import net.jxtz.jxtzos.entity.spider.SpiderLogServe;
import net.jxtz.jxtzos.service.spider.SpiderHousingInformationService;
import net.jxtz.jxtzos.spider.BaseSpiderProcessor;
import net.jxtz.jxtzos.spider.SpiderDefinition;
import net.jxtz.jxtzos.spider.SpiderFactory;
import net.jxtz.jxtzos.util.StampToDateAndDateToStamp;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;

/**
 * @author a123
 */
@Component
@Slf4j
public class HousingInformationProcessor implements BaseSpiderProcessor {

    private Integer updateCount;

    private Integer lastTime;

    public Integer getLastTime() {
        return lastTime;
    }

    public Integer getUpdateCount() {
        return updateCount;
    }

    public final static String SPIDER_NAME = "housingInformation";

    private final static int MAX_PAGE = 50;

    private Long spiderLogId = null;

    public HousingInformationProcessor() {
        SpiderFactory instance = SpiderFactory.getInstance();
        instance.putSpiderProcess(SPIDER_NAME, this);
        SpiderDefinition nanchangInformation = new SpiderDefinition("南昌新房", SPIDER_NAME);
        instance.putSpiderDefinition(SPIDER_NAME, nanchangInformation);
    }

    @Autowired
    private SpiderHousingInformationService service;

    private boolean process(Integer minPage) throws Exception {
        log.info("当前页：" + minPage);

        // 拿到首页的project_id 和 fullname
        List<NameValuePair> nameValuePairs = new ArrayList<>();
        nameValuePairs.add(new BasicNameValuePair("pageIndex", "" + minPage));
        // 首页翻页
        String indexPageUrl = "http://house.ncfdc.com.cn/House/houselistJson";
        String s = httpPost(indexPageUrl, nameValuePairs);
        JSONObject jsonObject = JSONObject.parseObject(s);

        JSONArray jsonObjectJSONArray = jsonObject.getJSONArray("list");
        if (jsonObjectJSONArray.size() == 0) {
            return false;
        }
        for (int i = 0; i < jsonObjectJSONArray.size(); i++) {
            // 进入详情页
            SpiderHousingInformation housingInformation = jsonObjectJSONArray.getObject(i, SpiderHousingInformation.class);
            log.info(housingInformation.toString());
            Integer project_id = housingInformation.getProject_id();
            String fullname = housingInformation.getFullname();
            String another_name = housingInformation.getAnotherName();

            // 拿到详情页总数
            List<NameValuePair> nameValuePairs1 = new ArrayList<>();
            nameValuePairs1.add(new BasicNameValuePair("project_id", project_id + ""));
            nameValuePairs1.add(new BasicNameValuePair("Key", ""));
            nameValuePairs1.add(new BasicNameValuePair("currentIndex", "1"));
            nameValuePairs1.add(new BasicNameValuePair("totalCount", "10"));

            String totalSize = httpPost(nameValuePairs1, project_id);

            if (totalSize == null || "0".equals(totalSize)) {
                // 如果总数为0，则跳过本次循环
                continue;
            }

            // 根据总页数算出分页总数
            int totalPage;
            int integer = Integer.parseInt(totalSize);

            if (integer % 10 == 0) {
                // 如果总页数能整除10则按个位数分页
                totalPage = integer / 10;
            } else {
                // 如果不能整除取个位数加1
                totalPage = (integer / 10) + 1;
            }

            // 开始翻页
            for (int j = 1; j <= totalPage; j++) {
                List<SpiderHousingInformation> housingInformations = new LinkedList<>();
                // 发送请求获取每页html
                List<NameValuePair> nameValuePairs2 = new ArrayList<>();
                nameValuePairs2.add(new BasicNameValuePair("project_id", project_id + ""));
                nameValuePairs2.add(new BasicNameValuePair("Key", ""));
                nameValuePairs2.add(new BasicNameValuePair("currentIndex", "" + j));
                nameValuePairs2.add(new BasicNameValuePair("totalCount", totalSize));

                // 详情页翻页开始
                String detailsUrl = "http://house.ncfdc.com.cn/House/idinfo_part";
                String s1 = httpPost(detailsUrl, nameValuePairs2);

                if (StringUtils.isBlank(s1)) {
                    continue;
                }
                Document parse = Jsoup.parse(s1);
                Elements floorAndCommunity = parse.getElementsByClass("ldinfo-card");
                floorAndCommunity.forEach(element -> {
                    String data = element.getElementsByClass("ldinfo-block").first().getElementsByTag("dl").get(1).getElementsByTag("dd").get(0).getElementsByTag("p").first().text();
                    log.info(data);
                    final String replace = data.replace("年", "-").replace("月", "-").replace("日", "");
                    final String[] split = replace.split("-");
                    String s2 = split[1];
                    if (s2.length() < 2) {
                        s2 = "0" + s2;
                    }
                    String s3 = split[2];
                    if (s3.length() < 2) {
                        s3 = "0" + s3;
                    }

                    String date = split[0] + "-" + s2 + "-" + s3;
                    if (StampToDateAndDateToStamp.isExpireOnMonth(date, 3)) {
                        log.info("过期数据，跳过");
                        return;
                    }
                    SpiderHousingInformation housingInformation1 = new SpiderHousingInformation();
                    housingInformation1.setAnotherName(another_name);
                    housingInformation1.setFullname(fullname);
                    housingInformation1.setProject_id(project_id);

                    String ld = element.getElementsByClass("ld-bar").first().getElementsByTag("h3").first().text();
                    // 备案小区名
                    String beianName = element.getElementsByClass("ld-bar").first().getElementsByTag("h3").get(0).getElementsByTag("span").get(0).text();
                    housingInformation1.setBeianName(beianName);


                    String floor = ld.substring(0, ld.indexOf(" ") + 1);
                    housingInformation1.setBuildingNumber(floor);

                    String preId = element.getElementsByClass("ldinfo-block").first().getElementsByTag("dl").get(0).getElementsByTag("dd").get(0).getElementsByTag("p").first().text();
                    String onSold = element.getElementsByClass("ldinfo-block").first().getElementsByTag("dl").get(0).getElementsByTag("dd").get(2).getElementsByTag("p").first().text();
                    String outSold = element.getElementsByClass("ldinfo-block").first().getElementsByTag("dl").get(0).getElementsByTag("dd").get(3).getElementsByTag("p").first().text();

                    String onSoldArea = onSold.substring(0, onSold.indexOf("/"));
                    String outSoldArea = outSold.substring(0, outSold.indexOf("/"));

                    String onSoldNumber = onSold.substring(onSold.indexOf("/") + 1);
                    String outSoldNumber = outSold.substring(outSold.indexOf("/") + 1);

                    housingInformation1.setPreId(preId);
                    housingInformation1.setOnSoldArea(onSoldArea);
                    housingInformation1.setOnSoldNumber(onSoldNumber);
                    housingInformation1.setOutSoldArea(outSoldArea);
                    housingInformation1.setOutSoldNumber(outSoldNumber);
                    housingInformation1.setDateOfIssue(data);
                    if (spiderLogId != null) {
                        housingInformation1.setSpiderLogId(spiderLogId);
                    }

                    // 放进集合
                    housingInformations.add(housingInformation1);
                });
                // 详情页翻页结束
                // 添加到数据库
                boolean b = service.saveBatch(housingInformations);
                log.info("存入数据库：" + b + "，housingInformations长度：" + housingInformations.size());
            }
        }

        return true;
    }

    /**
     *
     * 首页post请求
     * @param url 地址
     * @param nameValuePairs 请求参数
     * @return
     */
    @SneakyThrows
    private String httpPost(String url, List<NameValuePair> nameValuePairs) {
        TimeUnit.SECONDS.sleep(1);
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpPost httpPost = new HttpPost(url);
        httpPost.setConfig(RequestConfig.custom()
                .setConnectionRequestTimeout(1000 * 5)
                        .setConnectTimeout(1000 * 10)
                        .setSocketTimeout(5000 * 10)
                .build());
        httpPost.setHeader("Host","house.ncfdc.com.cn");
        httpPost.setHeader("Origin","http://house.ncfdc.com.cn");
        httpPost.setHeader("Referer","http://house.ncfdc.com.cn/House/houselist.html");
        httpPost.setHeader("Referer","http://house.ncfdc.com.cn/House/houselist.html");
        httpPost.setHeader("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36");
        CloseableHttpResponse response = null;
        String s = null;
        try {
            httpPost.setEntity(new UrlEncodedFormEntity(nameValuePairs, "utf-8"));
            response = httpClient.execute(httpPost);
            if (response.getStatusLine().getStatusCode() == 200) {
                log.info(url + "请求成功");
                s = EntityUtils.toString(response.getEntity());
            } else {
                log.error(url + "请求失败");
                return null;
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            if (httpClient != null) {
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return s;
    }

    /**
     *
     * 详情页post请求
     * @param nameValuePairs 请求参数
     * @param proId 项目id
     * @return response
     */
    private String httpPost(List<NameValuePair> nameValuePairs, Integer proId) throws Exception{
        TimeUnit.SECONDS.sleep(1);
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpPost httpPost = new HttpPost("http://house.ncfdc.com.cn/House/idinfo_partCount");
        httpPost.setConfig(RequestConfig.custom()
                .setConnectionRequestTimeout(1000 * 5)
                .setConnectTimeout(1000 * 10)
                .setSocketTimeout(5000 * 10)
                .build());
        CloseableHttpResponse response = null;
        String s = null;
        try {
            httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
            httpPost.setHeader("Host", "house.ncfdc.com.cn");
            httpPost.setHeader("Origin", "http://house.ncfdc.com.cn");
            httpPost.setHeader("User-Agent","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36");
            httpPost.setHeader("Referer", "http://house.ncfdc.com.cn/House/ldinfo.html?project_id=" + proId);
            httpPost.setHeader("X-Requested-With","XMLHttpRequest");
            httpPost.setEntity(new UrlEncodedFormEntity(nameValuePairs, "utf-8"));
            response = httpClient.execute(httpPost);
            if (response.getStatusLine().getStatusCode() == 200) {
                log.info(httpPost.getURI().getPath() + "请求成功");
                s = EntityUtils.toString(response.getEntity());
            } else {
                log.error(httpPost.getURI().getPath() + "请求失败");
                return null;
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }

            if (httpClient != null) {
                try {
                    httpClient.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
        return s;
    }

    @Override
    public void start(SpiderLogServe spiderLogServe, SpiderDefinition spiderDefinition) throws Exception{
        spiderLogId = spiderLogServe.getId();
        long startTime = System.currentTimeMillis() / 1000;
        int startCount = service.count();
        log.info("开始爬取");
        int minPage = 1;
        boolean flag = true;
        while (flag && minPage <= MAX_PAGE) {
            log.info("当前页：" + minPage);
            flag = process(minPage++);
            try {
                TimeUnit.SECONDS.sleep(3);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            if (!flag) break;
            log.info("爬取结果：" + flag);
        }
        long endTime = System.currentTimeMillis() / 1000;
        long time = endTime - startTime;
        int endCount = service.count();
        updateCount = endCount - startCount;
        lastTime = Long.valueOf(endTime).intValue();
        spiderLogServe.setCount(updateCount);
        log.info("爬取完毕，用时：" + time + "s，更新数据：" + updateCount);
    }
}
